import re
import threading

from bs4 import BeautifulSoup

from util import utils

baseUrl = 'https://www.t66y.com/thread0806.php?fid=20'
url = 'https://www.t66y.com/thread0806.php?fid=20&search=&page='
host = 'https://www.t66y.com/'
path = './text/'


def textFilter(content):
    if str(content).__contains__("</font>"):
        res = r'>(.*?)</font>'
    else:
        res = r'>(.*?)</a>'
    list = re.findall(res, str(content), re.S | re.M)
    if list.__len__() > 0:
        return str(list[0])
    return list


def spiderpage(pagetext):
    L = {}
    t1 = BeautifulSoup(pagetext, 'html.parser').find_all('a')
    for t2 in t1:
        url = str(t2.get('href'))
        if url.__contains__('htm_data'):
            title = textFilter(t2)
            if not (title.__contains__('color') or title.__contains__("\r")):
                L[host + url] = title
    return L


def spiderText(pagetext, fileName):
    myAttrs = {'class': 'tpc_content do_not_catch'}
    t1 = BeautifulSoup(pagetext, 'html.parser').find_all('div', myAttrs)
    s = str(t1)
    s = s.replace("<br/>", "\r")
    fo = open(fileName, "w",encoding='UTF-8')
    fo.write(s)
    fo.close()


def getText(urlList, name):
    print(urlList)
    utils.mkdir(name)
    for url in urlList:
        fileName = name + str(urlList[url]) + ".txt"
        print(fileName)
        spiderText(utils.GBK(url), fileName)


def run(index):
    if index == 1:
        getText(spiderpage(utils.GBK(baseUrl)), path)
    else:
        getText(spiderpage(utils.GBK(url + str(index))), path)


def loop():
    run(n)


if __name__ == '__main__':
    n = 0
    while 0 <= n < 100:
        t = threading.Thread(target=loop, name='LoopThread1')
        n = n + 1
        t.start()
